# coding:utf-8
import sqlite3
import requests
from random import choice
from scrapy import signals
"""
在爬虫文件中配置 middlewares

# 1、随机user-agent(SpiderPath是脚本文件的路径)
'DOWNLOADER_MIDDLEWARES': {
            'SpiderPath.downloadermiddlewares.UserAgentMiddleware': 543,
            # 禁用系统的middleware
            'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
        }

# 2、随机proxy
'DOWNLOADER_MIDDLEWARES': {
            'SpiderPath.downloadermiddlewares.ProxyMiddleware': 300,
        }


"""


# 封装随机user-agent
class RandAgent(object):
    agents = [
        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1',
        'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0',
        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
        'Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.9.168 Version/11.50',
        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; Tablet PC 2.0; .NET4.0E)',
        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; InfoPath.3)',
        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; GTB7.0)',
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
        'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)',
        'Mozilla/5.0 (Windows; U; Windows NT 6.1; ) AppleWebKit/534.12 (KHTML, like Gecko) Maxthon/3.0 Safari/534.12',
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E)',
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)',
        'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.33 Safari/534.3 SE 2.X MetaSr 1.0',
        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E)',
        'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1 QQBrowser/6.9.11079.201',
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E) QQBrowser/6.9.11079.201',
        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)',
    ]

    @classmethod
    def random(cls):
        from random import choice
        return choice(cls.agents)


# UerAgentMiddleware
class UserAgentMiddleware(object):
    """
    随机user-agent
    This middleware allows spiders to override the user_agent
    """

    def __init__(self, user_agent='Scrapy'):
        self.user_agent = user_agent

    @classmethod
    def from_crawler(cls, crawler):
        o = cls()
        crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
        return o

    # 重写什么也不做
    def spider_opened(self, spider):
        pass

    def process_request(self, request, spider):
        request.headers.setdefault(b'User-Agent', RandAgent.random())
        # TODO set cookie and referer, etc
        # request.headers.setdefault(b'Cookie', '')


# ProxyMiddleware
class ProxyMiddleware(object):
    __db_name = 'Proxies.sqlite'
    """
    随机Proxy
    This middleware allows spiders to override the proxy
    """

    def __init__(self, user_agent='Scrapy'):
        self.user_agent = user_agent

    @classmethod
    def from_crawler(cls, crawler):
        o = cls()
        crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
        return o

    # 重写什么也不做
    def spider_opened(self, spider):
        pass

    def process_request(self, request, spider):
        """
        本方法获取proxy的方式需要手动更改

        1、从数据库中获取proxy
        2、从网上的proxy的api接口中获取proxy
        3、其他方法
        :param request:
        :param spider:
        :return:
        """
        # 从proxy数据库中获取proxy
        # request.meta['proxy'] = ProxyMiddleware.__get_ip_from_sqlite(1)

        # 从网上找的api中获取proxy
        request.meta['proxy'] = ProxyMiddleware.__get_ip_from_api(1)

    @staticmethod
    def __get_ip_from_sqlite(num):
        """从proxy数据库中获取proxy"""
        print(f'第{num}次尝试获取proxy......')
        con = sqlite3.connect(ProxyMiddleware.__db_name)
        cursor = con.cursor()

        sql1 = """
            SELECT Ip_address,port FROM DaispiderItem;
        """

        # 查询
        cursor.execute(sql1)
        all_ips = cursor.fetchall()

        # 取一个ip，顺序取
        # 不使用随机数
        try:
            ip, port = all_ips[num]
            pro = f'http://{ip}:{port}'
        except Exception as e:
            raise ('代理池无法使用！请更换代理池')

        # 测试是否能用
        proxy = {
            'http': pro
        }
        response = requests.get('www.baidu.com', proxies=proxy)
        if response.text:
            # 输出代理
            print('使用的代理是：', pro)
            return f'http://{ip}:{port}'
        else:
            ProxyMiddleware.__get_ip_from_sqlite(num+1)

    @staticmethod
    def __get_ip_from_api(num):
        """从api中获取proxy，对于不同的api，本方法代码的实现不同，需要按实际情况重写本代码"""
        print(f'第{num}次尝试获取proxy......')
        # github上的一个api
        # http://118.24.52.95/get 获取一个随机的proxy
        proxy = requests.get("http://118.24.52.95/get/").json()
        proxy = proxy.get("proxy")

        # 测试代理的可用性
        response = requests.get('http://www.baidu.com', proxies={"http": "http://{}".format(proxy)})
        if response.text:
            print('使用的代理是：', proxy)
            return "http://{}".format(proxy)

        # 出错次数多了的话，重新获取
        ProxyMiddleware.__get_ip_from_api(num + 1)
